{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.99874914,  0.05000124,  0.9963569 ,  0.08528136, -0.04419665,\n",
       "        0.07410637], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('Acrobot-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAey0lEQVR4nO3de3BT170v8O/W00/JGLAUFzv4HmiIyyONCUbNzGlvUHGom+ZB5qQZbuqhTHJDBMMjwxS3gUzTzjWXzE2apAR6+wDmdhL3kFuSQiCJryGmGYQBg4kxwSU9UDuAZAK15KckS+v+QbxPBA7RxraWtvL9zOwZvPeS/RNYX/Zea6+1FSGEABFRkhlkF0BEX00MHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikkJa+GzatAmTJ09GRkYGysvLcfjwYVmlEJEEUsLnT3/6E1avXo1nn30Wx44dw6xZs1BRUYHOzk4Z5RCRBIqMiaXl5eW466678Otf/xoAEIvFUFRUhOXLl2Pt2rVf+vpYLIYLFy4gNzcXiqKMdblElCAhBLq7u1FYWAiD4cbnNqYk1aQKh8NoampCdXW1us9gMMDtdsPr9Q77mlAohFAopH59/vx5lJaWjnmtRHRzOjo6MGnSpBu2SXr4fPrpp4hGo3A4HHH7HQ4HTp8+Pexrampq8POf//y6/R0dHbDZbGNSJxFpFwwGUVRUhNzc3C9tm/TwuRnV1dVYvXq1+vXQG7TZbAwfohSUSHdI0sNnwoQJMBqN8Pv9cfv9fj+cTuewr7FarbBarckoj4iSJOmjXRaLBWVlZaivr1f3xWIx1NfXw+VyJbscIpJEymXX6tWrUVVVhdmzZ2POnDn41a9+hd7eXixevFhGOUQkgZTweeSRR3Dp0iWsX78ePp8Pd9xxB955553rOqGJKH1Juc9npILBIOx2OwKBADuciVKIls8m53YRkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSaA6fAwcO4L777kNhYSEURcGbb74Zd1wIgfXr1+OWW25BZmYm3G43zpw5E9fmypUrWLRoEWw2G/Ly8rBkyRL09PSM6I0Qkb5oDp/e3l7MmjULmzZtGvb4xo0b8fLLL2PLli1obGxEdnY2KioqMDAwoLZZtGgRWltbUVdXh927d+PAgQN44oknbv5dEJH+iBEAIHbu3Kl+HYvFhNPpFM8//7y6r6urS1itVvH6668LIYQ4deqUACCOHDmittm7d69QFEWcP38+oZ8bCAQEABEIBEZSPhGNMi2fzVHt8zl79ix8Ph/cbre6z263o7y8HF6vFwDg9XqRl5eH2bNnq23cbjcMBgMaGxuH/b6hUAjBYDBuIyJ9G9Xw8fl8AACHwxG33+FwqMd8Ph8KCgrijptMJuTn56ttrlVTUwO73a5uRUVFo1k2EUmgi9Gu6upqBAIBdevo6JBdEhGN0KiGj9PpBAD4/f64/X6/Xz3mdDrR2dkZd3xwcBBXrlxR21zLarXCZrPFbUSkb6MaPiUlJXA6naivr1f3BYNBNDY2wuVyAQBcLhe6urrQ1NSkttm3bx9isRjKy8tHsxwiSmEmrS/o6enBxx9/rH599uxZNDc3Iz8/H8XFxVi5ciV++ctfYurUqSgpKcG6detQWFiIBx54AABw++23495778Xjjz+OLVu2IBKJYNmyZfjhD3+IwsLCUXtjRJTitA6l7d+/XwC4bquqqhJCXB1uX7dunXA4HMJqtYp58+aJtra2uO9x+fJl8eijj4qcnBxhs9nE4sWLRXd3d8I1cKidKDVp+WwqQgghMftuSjAYhN1uRyAQYP8PUQrR8tnUxWgXEaUfhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFJonl6RSk6cOIGcnBzZZRDRZ7Qsh6zr8Ll06RL6+vpkl0FEn+nt7U24ra7Dx+12c3oFUQrRssoo+3yISAqGDxFJwfAhIikYPkQkBcOHiKRg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUDB8ikoLhQ0RSaAqfmpoa3HXXXcjNzUVBQQEeeOABtLW1xbUZGBiAx+PB+PHjkZOTg4ULF8Lv98e1aW9vR2VlJbKyslBQUIA1a9ZgcHBw5O+GiHRDU/g0NDTA4/Hg0KFDqKurQyQSwfz58+Mekbpq1Srs2rULO3bsQENDAy5cuICHHnpIPR6NRlFZWYlwOIyDBw9i+/bt2LZtG9avXz9674qIUp8Ygc7OTgFANDQ0CCGE6OrqEmazWezYsUNt89FHHwkAwuv1CiGE2LNnjzAYDMLn86ltNm/eLGw2mwiFQgn93EAgIACIQCAwkvKJaJRp+WyOqM8nEAgAAPLz8wEATU1NiEQicLvdaptp06ahuLgYXq8XAOD1ejFjxgw4HA61TUVFBYLBIFpbW4f9OaFQCMFgMG4jIn276fCJxWJYuXIl7r77bkyfPh0A4PP5YLFYkJeXF9fW4XDA5/OpbT4fPEPHh44Np6amBna7Xd2KioputmwiShE3HT4ejwcnT55EbW3taNYzrOrqagQCAXXr6OgY859JRGPLdDMvWrZsGXbv3o0DBw5g0qRJ6n6n04lwOIyurq64sx+/3w+n06m2OXz4cNz3GxoNG2pzLavVCqvVejOlElGK0nTmI4TAsmXLsHPnTuzbtw8lJSVxx8vKymA2m1FfX6/ua2trQ3t7O1wuFwDA5XKhpaUFnZ2dapu6ujrYbDaUlpaO5L0QkY5oOvPxeDx47bXX8NZbbyE3N1fto7Hb7cjMzITdbseSJUuwevVq5Ofnw2azYfny5XC5XJg7dy4AYP78+SgtLcVjjz2GjRs3wufz4ZlnnoHH4+HZDdFXiZZhNADDblu3blXb9Pf3i6eeekqMGzdOZGVliQcffFBcvHgx7vucO3dOLFiwQGRmZooJEyaIp59+WkQikYTr4FA7UWrS8tlUhBBCXvTdnGAwCLvdjkAgAJvNJrscIvqMls8m53YRkRQMHyKSguFDRFIwfIhICoYPEUnB8CEiKRg+RCQFw4eIpGD4EJEUNzWrnWg0CBFFNBpALNYPRTHBaLRBUTKgKIrs0igJGD6UdEIIRCLncenSFgQCuxAOd8BgyEZW1mwUFDyF3Nz/CkXhr2a6478wJZUQAqHQGZw7txi9vV5cnZsMRKP/RCDwCXp69qOw8H9g4sQnGEBpjn0+lFTRaADt7cvR23sQQ8Fz7fHz53+KQOAd6HDOM2nA8KGkCgTeRnd3/Q3bxGIB+P3/C7FY7w3bkb4xfCipeno+ABD90nZ9fccQjfIpJemM4UNJE4xG8WF/f8Lto8NcllH6YPhQ0gSjURzuVxKKlD5koPvLT5BIxxg+lFRvoxJB3HiFOwFgD+7FJzGuUpnOGD6UVP+BEvxvPIEBDP+wAAHgOL6J/yMWIRhLbm2UXLyRgpJGAQDFiP+LhQCAKmzHeHwKw2cXYmFY4MVc/Aqr4McEfBIOyyuWxhzDh5Imx2hEaUYGvL1R/Dv+DV64MBeHMAmfoBfZOIFZOIFZ6EcWIAQ+DoVkl0xjiOFDSWNWFOQZjQAAAQPacSvacavkqkgW9vlQ0hgBZBv4K0dX8TeBksakKMjSGD6cYpG+GD6UNAYAmRrCJ8zgSWsMH0oaRVGgZaWenmgUHG1PXwwfSlm9sRjDJ40xfChl9cRiiPHSK20xfCipMjT0+fxtYAARhk/aYvhQUs3JzoYxwbb+wUEMMnzSFsOHkspmMGjqdKb0xfChpMo1JnreQ+mO4UNJlWs08tE4BIDhQ0mWoSF4YkJggH0+aYvhQ0ml5awnKgR6Y7zTJ10xfChlRQH0RbmWarrSFD6bN2/GzJkzYbPZYLPZ4HK5sHfvXvX4wMAAPB4Pxo8fj5ycHCxcuBB+vz/ue7S3t6OyshJZWVkoKCjAmjVrMDg4ODrvhtIKz3zSm6bwmTRpEjZs2ICmpiYcPXoU99xzD+6//360trYCAFatWoVdu3Zhx44daGhowIULF/DQQw+pr49Go6isrEQ4HMbBgwexfft2bNu2DevXrx/dd0Upy6oosCV4o2FPLIZTAwNjXBHJoogRrlmQn5+P559/Hg8//DAmTpyI1157DQ8//DAA4PTp07j99tvh9Xoxd+5c7N27F9///vdx4cIFOBwOAMCWLVvwk5/8BJcuXYLFYhn2Z4RCIYQ+t6pdMBhEUVERAoEAbDYuMq4nXYOD+O6ZMzja15dQ+1eKirCsoGCMq6LREgwGYbfbE/ps3nSfTzQaRW1tLXp7e+FyudDU1IRIJAK32622mTZtGoqLi+H1egEAXq8XM2bMUIMHACoqKhAMBtWzp+HU1NTAbrerW1FR0c2WTZKZFEXTshqUvjT/FrS0tCAnJwdWqxVPPvkkdu7cidLSUvh8PlgsFuTl5cW1dzgc8Pl8AACfzxcXPEPHh459kerqagQCAXXr6OjQWjalCDPDhz6jeQ3n2267Dc3NzQgEAnjjjTdQVVWFhoaGsahNZbVaYbUO/6gV0heToiBTy70+uLqaIW9MTD+aw8disWDKlCkAgLKyMhw5cgQvvfQSHnnkEYTDYXR1dcWd/fj9fjidTgCA0+nE4cOH477f0GjYUBtKbwZcPftJVA+H2tPWiM9/Y7EYQqEQysrKYDabUV9frx5ra2tDe3s7XC4XAMDlcqGlpQWdnZ1qm7q6OthsNpSWlo60FNIBrWcwPbEYn9iepjSd+VRXV2PBggUoLi5Gd3c3XnvtNbz//vt49913YbfbsWTJEqxevRr5+fmw2WxYvnw5XC4X5s6dCwCYP38+SktL8dhjj2Hjxo3w+Xx45pln4PF4eFlFw+qORhk+aUpT+HR2duJHP/oRLl68CLvdjpkzZ+Ldd9/Fd7/7XQDAiy++CIPBgIULFyIUCqGiogKvvvqq+nqj0Yjdu3dj6dKlcLlcyM7ORlVVFZ577rnRfVeUNv42MICoEDCyzyftjPg+Hxm03EtAqWftJ5/gf15z5/sXmWq14sPSUk0rIJI8SbnPh+hmfT0jQ3YJlAIYPpR0uVzNkMDwIQm4miEBDB+SQEv4xAAuIp+mGD6UdCYNI1eDQqCfy2qkJYYPpTSGT/pi+FBKiwiBfl52pSWGDyWdlpGurmgUH3NBsbTE8KGkKzCZMMlsTqhtWAgEedmVlhg+lHS5RiNsHG7/ymP4UNJZFIXTJYjhQ8lnURRYNU4U1eEURPoSDB9KOrPG8OFQe3pi+FDSKdC2qFg3VzNMSwwfSnndPPNJS5rXcCYaXQJmRPBf8HfcinYoEOjAJPwdUxCCFYCCLj7RNi0xfEiKq4vICxShHf8dv4ELh5CDHgACvchBE+7EFizF3/Ev8Pb2QkDbzYmU+hg+JIUrOxsng6fwHJ7FdJyMCxYbuvEdNOBruIB1+AVCYqa0OmnssM+HpJhoAjzYdF3wDFEATMUZrMBLsCA0TAvSO575kBRzDGfQjw9ueCmlALgTx3AaxwHMSlJllCw88yEpCo19yEL/l7bLQAjTzH1JqIiSjeFDUmRpmF4xwcQT9HTE8KGkUxQFVzABAXz5Y496kYX/18/HI6Ujhg9JETR/HY3GBYjdoNcnBgUH8K84IUoR5tyutMPwISmKLZnYbXoKH+DuYQNIAGhCGTbBA/+ggouRSPKLpDHF8CEpbEYjBgwT8Euswx/x33ABtyACEwZhhA8O/Dv+Dc/i5/DBiU8iEXwc4nB7umFPHkkxNLn0CvLxCpbjT3gEBegEIPApJsIPB2IwYOi+5ogQEEJompBKqY3hQ1IYAHw7JwdNfX0QUOCHE344v7D9OZ75pB1edpE038jMTLjt3mBwDCshGRg+JIWiKMjUcK9PH5fVSDsMH5KmyGJBToIB1B2NopcBlFYYPiTNFKsV4xJ8ikXbwAD+EQ6PcUWUTAwfksZuNMKa6JlPLMZLrzTD8CFpFABant7VxbWc0wrDh6QxKwq+k5ubUFsB4GBPz9gWREk1ovDZsGEDFEXBypUr1X0DAwPweDwYP348cnJysHDhQvj9/rjXtbe3o7KyEllZWSgoKMCaNWswyHV6v3IMuNrvk6hTfGZ7Wrnp8Dly5Ah+85vfYObM+CUuV61ahV27dmHHjh1oaGjAhQsX8NBDD6nHo9EoKisrEQ6HcfDgQWzfvh3btm3D+vXrb/5dkC4pipJwnw8AhGIxDHKCadq4qfDp6enBokWL8Nvf/hbjxo1T9wcCAfz+97/HCy+8gHvuuQdlZWXYunUrDh48iEOHDgEA3nvvPZw6dQp//OMfcccdd2DBggX4xS9+gU2bNiH8BaMZoVAIwWAwbqP08HWrFZkJTploD4cRZL9P2rip8PF4PKisrITb7Y7b39TUhEgkErd/2rRpKC4uhtfrBQB4vV7MmDEDDodDbVNRUYFgMIjW1tZhf15NTQ3sdru6FRUV3UzZlIJuy8hAdoJnP60DA7jEy/O0oTl8amtrcezYMdTU1Fx3zOfzwWKxIC8vL26/w+GAz+dT23w+eIaODx0bTnV1NQKBgLp1dHRoLZtSlN1ohEnDZFFedqUPTRNLOzo6sGLFCtTV1SEjI2OsarqO1WqFVUPHJOmHAiQcPjEh0B4Oa5oTRqlL05lPU1MTOjs7ceedd8JkMsFkMqGhoQEvv/wyTCYTHA4HwuEwurq64l7n9/vhdF6dsex0Oq8b/Rr6eqgNfXVkGQz41wSH2wcBvMv+vrShKXzmzZuHlpYWNDc3q9vs2bOxaNEi9c9msxn19fXqa9ra2tDe3g6XywUAcLlcaGlpQWdnp9qmrq4ONpsNpaWlo/S2SC/MioJisznh9qFYDIKXXmlB02VXbm4upk+fHrcvOzsb48ePV/cvWbIEq1evRn5+Pmw2G5YvXw6Xy4W5c+cCAObPn4/S0lI89thj2LhxI3w+H5555hl4PB5eWn0FKYoCs4bh9n9Go4gIAQsXFdO9UV9M7MUXX4TBYMDChQsRCoVQUVGBV199VT1uNBqxe/duLF26FC6XC9nZ2aiqqsJzzz032qWQTszKzIRFURJaJP5gTw/+GY3CoSGwKDUpQofnsMFgEHa7HYFAADYbH6uid6f6+1F++jR6Epg4ajMYcOob38DXLJYkVEZaafls8r8Pks5mNCb8ixjD1bV9SP8YPiRdhsEAZ4KdzhEh8B9c1yctMHxIuhyDAaUJ3jcWEgJHenvHuCJKBoYPSWdVFBRoGG7vi8UQ019XJV2D4UPSKYoCo4ah80/CYU6zSAMMH0oJc7OzE17V8FBvLwYYPrrH8KGUUJqRAUOCZz+9sRgiDB/dY/hQSsgxGpHohVdYCFyKRMa0Hhp7DB9KCbkGA25JsNO5JxZDc3//GFdEY43hQylhnMmEf0lwbl9ECHQODnKCqc4xfCglZCoK8hN8gCAAXB4cBKNH3xg+lBK0Dref6OsDJ1noG8OHUsZ3cnISbts6MIAoL7t0jeFDKWNaZmbCI16DQqCfj0/WNYYPpYxEn2IBXJ3Z3sEJprrG8KGUMd5kSnh2++VolE8w1TmGD6WMApMJkzRMMB3ges66xvChlJFtMCBXw3D7OV526RrDh1KKWcNwe313N9jlrF8MH0op7gSf4QUAvdEobzTUMYYPpZSva3gSbk8shi6u56xbDB9KGYqiIEPDcPv5SARnOOKlWwwfSimTzGZMNCX2OLm+WAyBaJQjXjrF8KGU8jUN4QNcnWBK+sTwoZSSYzQiU8OlVyOfZKFbDB9KKQqg6TnsJ9nno1uj/qx2opFQAMy32eD93BmNAiDTYEDWZ9tEkwm3Wiy41WJBeXa2tFppZBg+lHLKsrLgzs3FZIsFJVYriiwWFJrNcJrNcJhMyDEaYVQUGHD11F3RcKZEqYPhQylFURRU2u34nt0OBVCX2GDApB+GD6WcRB+hQ/rGDmcikoLhQ0RSMHyISAqGDxFJwfAhIikYPkQkBcOHiKTQ5X0+Q0soBINByZUQ0ecNfSYTWeZEl+Fz+fJlAEBRUZHkSohoON3d3bDb7Tdso8vwyc/PBwC0t7d/6RtMNcFgEEVFRejo6IDNZpNdTsJYd3LptW4hBLq7u1FYWPilbXUZPobP1nux2+26+of5PJvNpsvaWXdy6bHuRE8I2OFMRFIwfIhICl2Gj9VqxbPPPgur1Sq7FM30WjvrTi691q2FIrj0PxFJoMszHyLSP4YPEUnB8CEiKRg+RCQFw4eIpNBl+GzatAmTJ09GRkYGysvLcfjwYan1HDhwAPfddx8KCwuhKArefPPNuONCCKxfvx633HILMjMz4Xa7cebMmbg2V65cwaJFi2Cz2ZCXl4clS5agp6dnTOuuqanBXXfdhdzcXBQUFOCBBx5AW1tbXJuBgQF4PB6MHz8eOTk5WLhwIfx+f1yb9vZ2VFZWIisrCwUFBVizZg0Gx/Axxps3b8bMmTPVu39dLhf27t2b0jUPZ8OGDVAUBStXrtRd7aNC6Extba2wWCziD3/4g2htbRWPP/64yMvLE36/X1pNe/bsET/72c/En//8ZwFA7Ny5M+74hg0bhN1uF2+++aY4ceKE+MEPfiBKSkpEf3+/2ubee+8Vs2bNEocOHRJ//etfxZQpU8Sjjz46pnVXVFSIrVu3ipMnT4rm5mbxve99TxQXF4uenh61zZNPPimKiopEfX29OHr0qJg7d6741re+pR4fHBwU06dPF263Wxw/flzs2bNHTJgwQVRXV49Z3X/5y1/E22+/Lf72t7+JtrY28dOf/lSYzWZx8uTJlK35WocPHxaTJ08WM2fOFCtWrFD366H20aK78JkzZ47weDzq19FoVBQWFoqamhqJVf2na8MnFosJp9Mpnn/+eXVfV1eXsFqt4vXXXxdCCHHq1CkBQBw5ckRts3fvXqEoijh//nzSau/s7BQARENDg1qn2WwWO3bsUNt89NFHAoDwer1CiKvBazAYhM/nU9ts3rxZ2Gw2EQqFklb7uHHjxO9+9ztd1Nzd3S2mTp0q6urqxLe//W01fPRQ+2jS1WVXOBxGU1MT3G63us9gMMDtdsPr9Uqs7IudPXsWPp8vrma73Y7y8nK1Zq/Xi7y8PMyePVtt43a7YTAY0NjYmLRaA4EAgP9cNaCpqQmRSCSu9mnTpqG4uDiu9hkzZsDhcKhtKioqEAwG0draOuY1R6NR1NbWore3Fy6XSxc1ezweVFZWxtUI6OPvezTpalb7p59+img0GvcXDwAOhwOnT5+WVNWN+Xw+ABi25qFjPp8PBQUFccdNJhPy8/PVNmMtFoth5cqVuPvuuzF9+nS1LovFgry8vBvWPtx7Gzo2VlpaWuByuTAwMICcnBzs3LkTpaWlaG5uTtmaAaC2thbHjh3DkSNHrjuWyn/fY0FX4UNjx+Px4OTJk/jggw9kl5KQ2267Dc3NzQgEAnjjjTdQVVWFhoYG2WXdUEdHB1asWIG6ujpkZGTILkc6XV12TZgwAUaj8bref7/fD6fTKamqGxuq60Y1O51OdHZ2xh0fHBzElStXkvK+li1bht27d2P//v2YNGmSut/pdCIcDqOrq+uGtQ/33oaOjRWLxYIpU6agrKwMNTU1mDVrFl566aWUrrmpqQmdnZ248847YTKZYDKZ0NDQgJdffhkmkwkOhyNlax8Lugofi8WCsrIy1NfXq/tisRjq6+vhcrkkVvbFSkpK4HQ642oOBoNobGxUa3a5XOjq6kJTU5PaZt++fYjFYigvLx+z2oQQWLZsGXbu3Il9+/ahpKQk7nhZWRnMZnNc7W1tbWhvb4+rvaWlJS486+rqYLPZUFpaOma1XysWiyEUCqV0zfPmzUNLSwuam5vVbfbs2Vi0aJH651StfUzI7vHWqra2VlitVrFt2zZx6tQp8cQTT4i8vLy43v9k6+7uFsePHxfHjx8XAMQLL7wgjh8/Lv7xj38IIa4Otefl5Ym33npLfPjhh+L+++8fdqj9m9/8pmhsbBQffPCBmDp16pgPtS9dulTY7Xbx/vvvi4sXL6pbX1+f2ubJJ58UxcXFYt++feLo0aPC5XIJl8ulHh8a+p0/f75obm4W77zzjpg4ceKYDv2uXbtWNDQ0iLNnz4oPP/xQrF27ViiKIt57772UrfmLfH60S2+1j5TuwkcIIV555RVRXFwsLBaLmDNnjjh06JDUevbv3y8AXLdVVVUJIa4Ot69bt044HA5htVrFvHnzRFtbW9z3uHz5snj00UdFTk6OsNlsYvHixaK7u3tM6x6uZgBi69atapv+/n7x1FNPiXHjxomsrCzx4IMPiosXL8Z9n3PnzokFCxaIzMxMMWHCBPH000+LSCQyZnX/+Mc/FrfeequwWCxi4sSJYt68eWrwpGrNX+Ta8NFT7SPF9XyISApd9fkQUfpg+BCRFAwfIpKC4UNEUjB8iEgKhg8RScHwISIpGD5EJAXDh4ikYPgQkRQMHyKS4v8DA/VmYPlJaAIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([ -1.        -1.        -1.        -1.       -12.566371 -28.274334], [ 1.        1.        1.        1.       12.566371 28.274334], (6,), float32)\n",
      "env.action_space= Discrete(3)\n",
      "state= [ 0.99933743  0.03639666  0.99858433 -0.0531914   0.00995294  0.0829151 ]\n",
      "action= 2\n",
      "next_state= [ 0.99980485  0.01975632  0.99998224  0.00595994 -0.1709635   0.49562523]\n",
      "reward= -1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7fa98ed77340>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-200.0, 0.0)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.A2C.Acrobot')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-75.4, 8.181686867633104)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/1.A2C.Acrobot')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAR8AAAEXCAYAAACUBEAgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgdUlEQVR4nO3df3BU5f0v8PfZn/m5G5KQXVISpYJgyo9q0LDaW3tLJNrUaqUz1uHS1HL1ShcGpMPUtIpTp3fCxbm12iJ2xltx5l6lg7dooaDNBAwqyw8j0YAa6Vc0ucAm/DC7SUh2N3s+948kRxYi7kI2zy55v2bOSM7z7O7nRM6bc57nnLOaiAiIiMaYSXUBRDQ+MXyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJZeGzfv16XH311cjIyEBFRQX279+vqhQiUkBJ+Pztb3/DqlWr8Pjjj+O9997DnDlzUFVVhc7OThXlEJECmoobSysqKnDjjTfiz3/+MwBA13WUlJRg+fLleOSRR7729bqu4/jx48jNzYWmackul4jiJCLo7u5GcXExTKaLH9tYxqgmQzgcRlNTE2pra411JpMJlZWV8Pl8I74mFAohFAoZPx87dgxlZWVJr5WILk17ezsmT5580T5jHj6nTp1CNBqFy+WKWe9yufDxxx+P+Jq6ujr87ne/u2B9e3s7HA5HUuokosQFg0GUlJQgNzf3a/uOefhcitraWqxatcr4eXgDHQ4Hw4coBcUzHDLm4VNYWAiz2YyOjo6Y9R0dHXC73SO+xm63w263j0V5RDRGxny2y2azoby8HA0NDcY6XdfR0NAAj8cz1uUQkSJKTrtWrVqFmpoazJ07FzfddBP++Mc/ore3F/fff7+KcohIASXhc++99+LkyZNYs2YN/H4/vv3tb+P111+/YBCaiK5cSq7zuVzBYBBOpxOBQIADzkQpJJF9k/d2EZESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESiQcPrt378add96J4uJiaJqGV199NaZdRLBmzRpMmjQJmZmZqKysxJEjR2L6nDlzBosWLYLD4UBeXh6WLFmCnp6ey9oQIkovCYdPb28v5syZg/Xr14/Yvm7dOjzzzDN47rnnsG/fPmRnZ6Oqqgr9/f1Gn0WLFuHw4cOor6/Htm3bsHv3bjz44IOXvhVElH7kMgCQLVu2GD/rui5ut1uefPJJY11XV5fY7XZ5+eWXRUTkww8/FABy4MABo8+OHTtE0zQ5duxYXJ8bCAQEgAQCgcspn4hGWSL75qiO+Rw9ehR+vx+VlZXGOqfTiYqKCvh8PgCAz+dDXl4e5s6da/SprKyEyWTCvn37RnzfUCiEYDAYsxBRehvV8PH7/QAAl8sVs97lchltfr8fRUVFMe0WiwX5+flGn/PV1dXB6XQaS0lJyWiWTUQKpMVsV21tLQKBgLG0t7erLomILtOoho/b7QYAdHR0xKzv6Ogw2txuNzo7O2PaBwYGcObMGaPP+ex2OxwOR8xCROltVMNnypQpcLvdaGhoMNYFg0Hs27cPHo8HAODxeNDV1YWmpiajz86dO6HrOioqKkazHCJKYZZEX9DT04N///vfxs9Hjx5Fc3Mz8vPzUVpaipUrV+L3v/89pk2bhilTpuCxxx5DcXEx7r77bgDAddddh9tvvx0PPPAAnnvuOUQiESxbtgw//elPUVxcPGobRkQpLtGptF27dgmAC5aamhoRGZxuf+yxx8Tlcondbpf58+dLa2trzHucPn1a7rvvPsnJyRGHwyH333+/dHd3x10Dp9qJUlMi+6YmIqIw+y5JMBiE0+lEIBDg+A9RCklk30yL2S4iuvIwfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUiLh2ytSyfvvv4+cnBzVZRDRkEQeh5zW4XPy5EmcPXtWdRlENKS3tzfuvmkdPpWVlby9giiFJPKUUY75EJESDB8iUoLhQ0RKMHyISAmGDxEpkdazXUTpQheBjsF/7TUAmqYprkg9hg/RGNjb24tHjx3DzMxMfCszE1Ptdlxjt6PIYoFF02DRNJjGWSAxfIjGwD+6urCrpwe7enqgAcgymZBtMsFttaIsIwPXZWRg+tBybUYGskxX/ogIw4coyXQRfNjfb/wsAHp1Hb26js6BAXzQ1wcAsGka7JqGZ0pK8PPCQkXVjp0rP16JFDsTjeKzUOhr+4WHxoWustmSX1QKYPgQJZk/EsHHcYQPADjNZszOykpyRamB4UOUZEdDIehxfkOVy2JBxjgZeGb4ECWRiKA+GEQ0zv63ORzIHAeDzQDDhyipQiL4PByOu3+BxTJuptwZPkRJdHJgALvjfMBWjsmEBePoETEMH6IkCkajCMc53mPVNLit1iRXlDoYPkRJ9HowiH5dj6tveVYWcsfJeA/A8CFKGl0Ex8NhxBc9QFlm5ri4snnY+NlSojHWp+t4N85njFsAeLKzx9UNpwwfoiQ5q+sxt1VcjFnTMDszM8kVpRaGD1GSfBoKIRTneM83rFbkW8bXrZYMH6IkEBG809uL7jjDZ05WFgoYPkR0uXQAxxO4uHCS1TruHjHB8CFKgrAI6ru74+prAvBDp3NcDTYDCYZPXV0dbrzxRuTm5qKoqAh33303WltbY/r09/fD6/WioKAAOTk5WLhwITo6OmL6tLW1obq6GllZWSgqKsLq1asxMDBw+VtDlCK6o1GcifPvtAmDRz7jTULh09jYCK/Xi71796K+vh6RSAQLFiyI+YrUhx9+GFu3bsXmzZvR2NiI48eP45577jHao9EoqqurEQ6HsWfPHrz44ovYuHEj1qxZM3pbRaTY+319OBON73bSb2VmonScPMPnXJpInNd+j+DkyZMoKipCY2Mjvvvd7yIQCGDixIl46aWX8JOf/AQA8PHHH+O6666Dz+fDvHnzsGPHDvzwhz/E8ePH4XK5AADPPfccfv3rX+PkyZOwxfE/IRgMwul0IhAI8OuSKeWICP5y6hSWtrXF1b/K4cCOqVOviNOuRPbNyxrzCQQCAID8/HwAQFNTEyKRCCorK40+M2bMQGlpKXw+HwDA5/Nh1qxZRvAAQFVVFYLBIA4fPjzi54RCIQSDwZiFKFXpALYP7RvxKM/KuiKCJ1GXHD66rmPlypW45ZZbMHPmTACA3++HzWZDXl5eTF+XywW/32/0OTd4htuH20ZSV1cHp9NpLCUlJZdaNlHSRUXiemzqsO/m5CSxmtR1yeHj9Xpx6NAhbNq0aTTrGVFtbS0CgYCxtLe3J/0ziS7Vp+EwOuMcbM4ymcbd9T3DLmmrly1bhm3btmH37t2YPHmysd7tdiMcDqOrqyvm6KejowNut9vos3///pj3G54NG+5zPrvdDrvdfimlEo0pEcGR/n50xBk+19rt+NY4u61iWEJHPiKCZcuWYcuWLdi5cyemTJkS015eXg6r1YqGhgZjXWtrK9ra2uDxeAAAHo8HLS0t6OzsNPrU19fD4XCgrKzscraFKCV8msApV67ZPG6e2Xy+hI58vF4vXnrpJbz22mvIzc01xmicTicyMzPhdDqxZMkSrFq1Cvn5+XA4HFi+fDk8Hg/mzZsHAFiwYAHKysqwePFirFu3Dn6/H48++ii8Xi+PbuiKsDWBweYfnzc+Op4kFD4bNmwAAHzve9+LWf/CCy/g5z//OQDgqaeegslkwsKFCxEKhVBVVYVnn33W6Gs2m7Ft2zYsXboUHo8H2dnZqKmpwRNPPHF5W0KUAs7qOr6I8/oeM4Cp4/gf3Mu6zkcVXudDqar57Fl8p7UVvXHcUDrZasU706ej9AoKoDG7zoeIYnVGInEFDzA43uMah7dVDGP4EI2it+L8pgoAuDk7G5ZxOtgMMHyIRk1UBAf7+uLu/+2srHG9A47nbScaVccjERyKM3wyNA3ftNvH5W0Vwxg+RKPki4EBHIvzAWITzGaUZ2UluaLUxvAhGiWH+vri/pqcPIsFOePoa3JGMr63nmiUiAgaurvjDp8FDgcyGD5EdLnCIjiVwNM4XRbLuN/5xvv2E42KzoEBvB3nNHu2yYT/lJMzrgebgUu8q51oNIhEEY0GoOt9iMKMqCkXNlMmrJqWdjtmIBpFX5wXF9o0DddkZCS5otTH8KExJyKIRI7h5MnnEAhsRSjcjj5kIGj9Nj7MXIwz9ptxjT0b12ZkYJLVCqfZjFyTCbYUHiPZGQwiFOedStMzMpCdwtsyVhg+NKZEBKHQEXz22f3o7fUBGNxh7QAKoydwQ/9bWA8vnsQ90GGGw2xGic2GYqsV19jtuC4jAzOGQqnAYsEEiyXmL7GKIyZdBEfD4bgHm8uzssb9TBfA8KExFo0G0Na2HL29ey5o0wDkoAderMcJuPEOvoMz0SjO9PXh/XMu3jMBcJrNKLJY4LZaMXUolKZlZGCyzQa3xYJCiwUWTcNwFCUzlPp0He+fPRtXXw2DX5NjSrPTymRg+NCYCgT+ie7uhq9s1wDkogeL8b/xHsrRhwsvxNMBfBGN4otoFK2hEBqHBno1AA6zGflmMwotFkyz2zEjMxPX2u0osdkw2WbDxKFQMg/1H41Q6tV1fBDnlc02TUO103nZn3klYPjQmOrpeRvA1z/vZgY+RjZ6RwyfryIYHPgNRKM4Gg7jwNmzwBdfwAQgx2RCzjmhNM1uxzV2O6bY7Zhis6HQaoVN02DVNJiQWCh9Fg4jHOd4j1XT4DSb437vKxnDh654OoCgriOo6zgeiRhHKSYAmSYTMjQNhRYLvjkUSNfY7Zg69N8CiwVZJhMyTSaYRwgkEcE7PT3oiXOm65acHLjH8WM0zsXwoTFlNufG1a8PmYgiuUcIOgZPmXoBnB46hQMGQ8k6dBRUYLHgmzYbSoeOlqbb7ZiakYFCiwW5ZjMyNQ1H+vsR7xP5Sm22cfvM5vMxfGhMFRQsxqlT/wvR6Jmv7CMAXkcVAlAzNqIDCIkgJIKecBifh8NATw80fBlMEy0WfGNocHtnd/c5lZ/vy6AxAbgtNzftrmFKFoYPjamMjDJMmvQ4jh17BCIXDtIKgIO4Hv8H/wV6ko98EiUYHK2KiqA9EkF7JGK0ZKEXt2APbsY7KMRpdKIIb+E72It56EcmgMGZN15c+CWGD40pTTNj4sT/BgDo6PgfCEf8wNAVMmHY4MM8/BEP4xQmKqwyEYJJOIHVeBIe+GDBl/d33YEdaMSt+J/4FU6hENfYM1DM8R4Dw4fGnMlkR1GRF05nFXb7X8PbX3yAgGThXZmNg5iT0AyXag4E8QjWwgMfTOeddtkQwXw0wAQdv8PjuMrmwMRx+u2kI+FvgpTQNDMyMqbj5pJfYWJRCJ+GQigJhTAnHMbR0ODPZ6JR9Os6+nQ9jsl5FQTV+CfmYe8FwTPMBMH38CbewS24KuO/jjhjNl4xfEipHLMZ12dl4fqsLIgIogAGRIxHVLQPDfgeDYXw71AInwx9FXFPNIpuXY/7+ppkMEHHD7Ad5q+5scIMHT/ENkxyPDxGlaUHhg+lDE3TYAFg0TRkYPBq5W/a7RARCAZHhqIiCEajOB6J4Fgkgs9DIbQOhdL/C4dxJhrFFwMDODtGoXTuGM/FZGs6Jlo43nMuhg+lPG3oHi0TBoNposmEiVYrZp8TMILBbwvtHBhAZySC9nAYn4RCaO3vx9FwGJ2RCDoHBhCIRuO+Jmc0XZuRgWszOdN1LoYPpa1zr5fRMHgKlzN0tFRxTjCFRQbvBRsYwIlIBEdCIRzp78d/hEI4NnQEdWpgAPrQaV+84aTDhEbciqn4N75uJGdy/kLkmlLr0gHVGD50RTo3mOyaBrfJBLfViusyM/F9wBhf6olGEdR1nB4YwKehEI6EQviPUAhtoRDawmH4BwbQr+sYEBnhBEvDFvwYN2MPrsNHXxlAOTnfQUHBz3hx4XkYPjQuDY8v5VksyMPgbQ/XD32VTVQE/bqO/qHxpc/CYXw6FEqfhUI4Gg6jPRxGr64joE9CnfwGv8F/x7X45LzBZzOysytQWroBFkuRgq1MbQwfovOYNQ3ZZjOyARRYLJhit+M/5+YOPoFxaOkTwTFjJu4baAvPgLnv/2JyuBETcAZW6yQ4ndUoKFgEi8XNo54RMHyI4qRpGmyaBhuAbACFFgvmDF0ioKMIItdDQxQadAAmaJoFmsYnFn4Vhg/RZdKGHk4GzQyk2P1oqYyxTERKMHyISAmGDxEpwfAhIiUYPkSkRELhs2HDBsyePRsOhwMOhwMejwc7duww2vv7++H1elFQUICcnBwsXLgQHR0dMe/R1taG6upqZGVloaioCKtXr8bAQHw35xHRlSOh8Jk8eTLWrl2LpqYmvPvuu/j+97+Pu+66C4cPHwYAPPzww9i6dSs2b96MxsZGHD9+HPfcc4/x+mg0iurqaoTDYezZswcvvvgiNm7ciDVr1ozuVhFR6pPLNGHCBHn++eelq6tLrFarbN682Wj76KOPBID4fD4REdm+fbuYTCbx+/1Gnw0bNojD4ZBQKPSVn9Hf3y+BQMBY2tvbBYAEAoHLLZ+IRlEgEIh737zkMZ9oNIpNmzaht7cXHo8HTU1NiEQiqKysNPrMmDEDpaWl8Pl8AACfz4dZs2bB5XIZfaqqqhAMBo2jp5HU1dXB6XQaS0lJyaWWTUQpIuHwaWlpQU5ODux2Ox566CFs2bIFZWVl8Pv9sNlsyMvLi+nvcrng9/sBAH6/PyZ4htuH275KbW0tAoGAsbS3tydaNhGlmIRvr5g+fTqam5sRCATwyiuvoKamBo2NjcmozWC322G325P6GUQ0thIOH5vNhqlTpwIAysvLceDAATz99NO49957EQ6H0dXVFXP009HRAbfbDQBwu93Yv39/zPsNz4YN9yGi8eGyr/PRdR2hUAjl5eWwWq1oaGgw2lpbW9HW1gaPxwMA8Hg8aGlpQWdnp9Gnvr4eDocDZWVll1sKEaWRhI58amtrcccdd6C0tBTd3d146aWX8Oabb+KNN96A0+nEkiVLsGrVKuTn58PhcGD58uXweDyYN28eAGDBggUoKyvD4sWLsW7dOvj9fjz66KPwer08rSIaZxIKn87OTvzsZz/DiRMn4HQ6MXv2bLzxxhu47bbbAABPPfUUTCYTFi5ciFAohKqqKjz77LPG681mM7Zt24alS5fC4/EgOzsbNTU1eOKJJ0Z3q4go5WkiCr/46BIFg0E4nU4EAgE4HA7V5RDRkET2Td7bRURKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKXFZ4bN27VpomoaVK1ca6/r7++H1elFQUICcnBwsXLgQHR0dMa9ra2tDdXU1srKyUFRUhNWrV2NgYOBySiGiNHPJ4XPgwAH85S9/wezZs2PWP/zww9i6dSs2b96MxsZGHD9+HPfcc4/RHo1GUV1djXA4jD179uDFF1/Exo0bsWbNmkvfCiJKP3IJuru7Zdq0aVJfXy+33nqrrFixQkREurq6xGq1yubNm42+H330kQAQn88nIiLbt28Xk8kkfr/f6LNhwwZxOBwSCoVG/Lz+/n4JBALG0t7eLgAkEAhcSvlElCSBQCDuffOSjny8Xi+qq6tRWVkZs76pqQmRSCRm/YwZM1BaWgqfzwcA8Pl8mDVrFlwul9GnqqoKwWAQhw8fHvHz6urq4HQ6jaWkpORSyiaiFJJw+GzatAnvvfce6urqLmjz+/2w2WzIy8uLWe9yueD3+40+5wbPcPtw20hqa2sRCASMpb29PdGyiSjFWBLp3N7ejhUrVqC+vh4ZGRnJqukCdrsddrt9zD6PiJIvoSOfpqYmdHZ24oYbboDFYoHFYkFjYyOeeeYZWCwWuFwuhMNhdHV1xbyuo6MDbrcbAOB2uy+Y/Rr+ebgPEV35Egqf+fPno6WlBc3NzcYyd+5cLFq0yPiz1WpFQ0OD8ZrW1la0tbXB4/EAADweD1paWtDZ2Wn0qa+vh8PhQFlZ2ShtFhGluoROu3JzczFz5syYddnZ2SgoKDDWL1myBKtWrUJ+fj4cDgeWL18Oj8eDefPmAQAWLFiAsrIyLF68GOvWrYPf78ejjz4Kr9fLUyuicSSh8InHU089BZPJhIULFyIUCqGqqgrPPvus0W42m7Ft2zYsXboUHo8H2dnZqKmpwRNPPDHapRBRCtNERFQXkahgMAin04lAIACHw6G6HCIaksi+yXu7iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkBMOHiJRg+BCREgwfIlKC4UNESjB8iEgJhg8RKcHwISIlGD5EpATDh4iUYPgQkRIMHyJSguFDREowfIhICYYPESnB8CEiJRg+RKQEw4eIlLCoLuBSiAgAIBgMKq6EiM41vE8O76MXk5bhc/r0aQBASUmJ4kqIaCTd3d1wOp0X7ZOW4ZOfnw8AaGtr+9oNTDXBYBAlJSVob2+Hw+FQXU7cWPfYSte6RQTd3d0oLi7+2r5pGT4m0+BQldPpTKv/MedyOBxpWTvrHlvpWHe8BwQccCYiJRg+RKREWoaP3W7H448/DrvdrrqUhKVr7ax7bKVr3YnQJJ45MSKiUZaWRz5ElP4YPkSkBMOHiJRg+BCREgwfIlIiLcNn/fr1uPrqq5GRkYGKigrs379faT27d+/GnXfeieLiYmiahldffTWmXUSwZs0aTJo0CZmZmaisrMSRI0di+pw5cwaLFi2Cw+FAXl4elixZgp6enqTWXVdXhxtvvBG5ubkoKirC3XffjdbW1pg+/f398Hq9KCgoQE5ODhYuXIiOjo6YPm1tbaiurkZWVhaKioqwevVqDAwMJK3uDRs2YPbs2cbVvx6PBzt27Ejpmkeydu1aaJqGlStXpl3to0LSzKZNm8Rms8lf//pXOXz4sDzwwAOSl5cnHR0dymravn27/Pa3v5W///3vAkC2bNkS07527VpxOp3y6quvyvvvvy8/+tGPZMqUKdLX12f0uf3222XOnDmyd+9eeeutt2Tq1Kly3333JbXuqqoqeeGFF+TQoUPS3NwsP/jBD6S0tFR6enqMPg899JCUlJRIQ0ODvPvuuzJv3jy5+eabjfaBgQGZOXOmVFZWysGDB2X79u1SWFgotbW1Sav7H//4h/zzn/+UTz75RFpbW+U3v/mNWK1WOXToUMrWfL79+/fL1VdfLbNnz5YVK1YY69Oh9tGSduFz0003idfrNX6ORqNSXFwsdXV1Cqv60vnho+u6uN1uefLJJ411XV1dYrfb5eWXXxYRkQ8//FAAyIEDB4w+O3bsEE3T5NixY2NWe2dnpwCQxsZGo06r1SqbN282+nz00UcCQHw+n4gMBq/JZBK/32/02bBhgzgcDgmFQmNW+4QJE+T5559Pi5q7u7tl2rRpUl9fL7feeqsRPulQ+2hKq9OucDiMpqYmVFZWGutMJhMqKyvh8/kUVvbVjh49Cr/fH1Oz0+lERUWFUbPP50NeXh7mzp1r9KmsrITJZMK+ffvGrNZAIADgy6cGNDU1IRKJxNQ+Y8YMlJaWxtQ+a9YsuFwuo09VVRWCwSAOHz6c9Jqj0Sg2bdqE3t5eeDyetKjZ6/Wiuro6pkYgPX7foymt7mo/deoUotFozC8eAFwuFz7++GNFVV2c3+8HgBFrHm7z+/0oKiqKabdYLMjPzzf6JJuu61i5ciVuueUWzJw506jLZrMhLy/vorWPtG3DbcnS0tICj8eD/v5+5OTkYMuWLSgrK0Nzc3PK1gwAmzZtwnvvvYcDBw5c0JbKv+9kSKvwoeTxer04dOgQ3n77bdWlxGX69Olobm5GIBDAK6+8gpqaGjQ2Nqou66La29uxYsUK1NfXIyMjQ3U5yqXVaVdhYSHMZvMFo/8dHR1wu92Kqrq44bouVrPb7UZnZ2dM+8DAAM6cOTMm27Vs2TJs27YNu3btwuTJk431brcb4XAYXV1dF619pG0bbksWm82GqVOnory8HHV1dZgzZw6efvrplK65qakJnZ2duOGGG2CxWGCxWNDY2IhnnnkGFosFLpcrZWtPhrQKH5vNhvLycjQ0NBjrdF1HQ0MDPB6Pwsq+2pQpU+B2u2NqDgaD2Ldvn1Gzx+NBV1cXmpqajD47d+6EruuoqKhIWm0igmXLlmHLli3YuXMnpkyZEtNeXl4Oq9UaU3trayva2tpiam9paYkJz/r6ejgcDpSVlSWt9vPpuo5QKJTSNc+fPx8tLS1obm42lrlz52LRokXGn1O19qRQPeKdqE2bNondbpeNGzfKhx9+KA8++KDk5eXFjP6Pte7ubjl48KAcPHhQAMgf/vAHOXjwoHz++eciMjjVnpeXJ6+99pp88MEHctddd4041X799dfLvn375O2335Zp06Ylfap96dKl4nQ65c0335QTJ04Yy9mzZ40+Dz30kJSWlsrOnTvl3XffFY/HIx6Px2gfnvpdsGCBNDc3y+uvvy4TJ05M6tTvI488Io2NjXL06FH54IMP5JFHHhFN0+Rf//pXytb8Vc6d7Uq32i9X2oWPiMif/vQnKS0tFZvNJjfddJPs3btXaT27du0SABcsNTU1IjI43f7YY4+Jy+USu90u8+fPl9bW1pj3OH36tNx3332Sk5MjDodD7r//funu7k5q3SPVDEBeeOEFo09fX5/88pe/lAkTJkhWVpb8+Mc/lhMnTsS8z2effSZ33HGHZGZmSmFhofzqV7+SSCSStLp/8YtfyFVXXSU2m00mTpwo8+fPN4InVWv+KueHTzrVfrn4PB8iUiKtxnyI6MrB8CEiJRg+RKQEw4eIlGD4EJESDB8iUoLhQ0RKMHyISAmGDxEpwfAhIiUYPkSkxP8HBbwmbiZHF4EAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-70.0 71 [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, -1.0, 0.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        action = action.item()\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 2 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
